/**
  * Created by Joshua Conero(O'Uymas) on 2016/10/21 0021.
  * QQ: 957376074
  * E-mail: Brximl@163.com
  * 爬虫的实现以及与爬虫相关的处理
  */
import java.io.{File, PrintWriter}
import java.net._
import scala.collection.JavaConversions._
import org.jsoup.Jsoup
import scala.io._
import scala.collection.mutable.ArrayBuffer
import scala.util.control.Breaks._
import scala.xml.XML
class Crawler(url:String="") {
  // 构造函数
  val u = url
  var crawled = new ArrayBuffer[String]()   // 已经爬取的URL列表
  var crawling = new ArrayBuffer[String]()  // 等待爬取的URL列表
  var crawlCount = 0                        // 爬取次数
  var crawlLog = ""                         // 爬取日志文件
  var crawlRunTimeDir = ""                  // 爬取日志目录
  val helper = new Helper

  // java/net 获取网页内容
  def getBody(iptUrl: String=""):Unit = {
    var urlString = iptUrl
    if(iptUrl == ""){
      urlString = u
    }
    val url = new URL(urlString)
    val host = url.getHost()
    var port = url.getPort().toString()
    var path = url.getPath()
    if(path == "/" || path == "\\") path = "index"
    else{
      path = path.replaceFirst("/","")
      path = path.replaceAll("/","_")
    }
    //println(urlString,"请求地址")
    val ctt = Source.fromURL(urlString,"utf-8").mkString
    val dir = getRunTimeDir()
    val f = new File(dir)
    f.mkdirs()
    val pw = new PrintWriter(dir+path+".html")
    val doc = Jsoup.parse(ctt)
    val links = doc.select("a[href]")
    var href = ""
    var text = ""
    var newUrl = ""
    val Arr = crawled.toArray
    breakable {
      for (v <- links) {
        href = v.attr("href")
        // 非法检测
        if(href == "#" || href.indexOf("javascript:") > -1 || href.indexOf("Javascript:") > -1 || href.indexOf("JavaScript") > -1) break()
        if(isNotWebScript(href)) break()
        text = v.text()
        if (href.indexOf("http") == -1 || href.indexOf(host) > -1) {
          if (href.indexOf(host) > -1) newUrl = href
          else{
            newUrl = "http://" + host
            if(port == "-1" || port == "") newUrl += ""
            else if(port != "80") newUrl += ":"+ port
            newUrl += href
          }
          if (Arr.indexOf(newUrl) > -1) break()
          crawlCount += 1
          crawled.append(newUrl)
          println(newUrl)
          report(text+"=>"+newUrl+";（次数"+crawlCount+"）\r\n",dir+helper.stamp()+".log")
          try{
            this.getBody(newUrl)
          }
        } else {
          println("本次总共访问" + host + "的次数", crawlCount, ":本次请求地址=", href)
        }
      }
    }
    pw.write(ctt)
    pw.close()
  }
  // 利用jsoup 封装方法获取网络内容
  def getHtml(iptUrl:String=""):Unit = {
    var uString = iptUrl
    if(iptUrl == "") uString = u
    if(uString == "") return
    val doc = Jsoup.connect(uString).get()
    println(doc.title())
  }
  // 发送信息
  def sendData():Unit = {
    var url = "http://127.0.0.1:9992/wjtai/web"
    url = "http://127.0.0.1:9992/wjtai/test/head.html"
    val doc = Jsoup.connect(url)
      .data("scala","Emma")
      .timeout(3000)
      .post()
    println(doc.body())
  }
  // 日志登记
  def report(ctt:String,fname:String=""):Unit = {
    if(fname != "" && crawlLog == "") crawlLog = fname
    if(crawlLog == "") return
    helper.file_append(crawlLog,ctt)
  }
  // 生成日志目录
  def getRunTimeDir(urlIpt:String=""):String = {
    var retDir = crawlRunTimeDir
    if(retDir == "") {
      var newUrl = urlIpt
      if (newUrl == "") newUrl = u
      retDir = "D:/Joshua/Active/Scala/app/Scala_start/runtime/"
      val url = new URL(newUrl)
      retDir += url.getHost()
      var path = url.getPath()
      if(path != "/"){
        retDir += path.replaceAll("/","_")
      }
      retDir += "/"
      crawlRunTimeDir = retDir
    }
    return retDir
  }
  // 检测网页不是一个有效的网页
  def isNotWebScript(url:String):Boolean = {
    val str = url.toLowerCase()
    if(str.indexOf(".apk") > -1) return true
    return false
  }
  // 爬取安居客的房源信息
  def anjuke():Unit ={

    var tgt = "http://gy.fang.anjuke.com/loupan/all/p"
    var iptTgt = ""
    val max = 10 // 实际查看10页
    val yunxing = (u:String) =>{
      val doc = Jsoup.connect(u).get()
      val divs = doc.select("div.item-mod")
      var name = ""
      var state = ""
      var price = ""
      var address = ""
      var tel = ""
      var hstype = ""
      var sqlAdd1 = ""
      var ctt = 1
      val uid = "20161108-ajk-2"
      val md = new model("com.mysql.jdbc.Driver","jdbc:mysql://localhost/zaueen?useUnicode=true&characterEncoding=UTF-8","root", "151001")
      for (v <- divs) {
        name = v.select("a.items-name").text()
        if(name != "") {
          state = v.select("i").text() + v.select("a.list-dp").text()
          price = v.select("p.price > span").text()
          if(price == ""){
            price = "0"
          }
          address = v.select("p.address > a").text()
          tel = v.select("p.tel").text()
          hstype = v.select("div.infos > p:not(class)").text()
          sqlAdd1 = "insert into `anjuke`(name,price,state,address,tel,remark,scala_uid) values('"+name+"','"+price+"','"+state+"','"+address+"','"+tel+"','"+hstype+"','"+uid+"')"

          //println(sqlAdd1)
          //return

          md.update(sqlAdd1)
          ctt = ctt + 1
        }
      }
      val sqlAdd = "insert into `scala`(uid,url,meta_ctt,remark) values('"+uid+"','"+u+"','"+ctt+"','首次数据抓包保存结果')"
      md.update(sqlAdd)
      //println(sqlAdd)

    }
    for(i<- 1 to max){
      iptTgt = tgt+i.toString()+"/"
      //println(iptTgt)
      yunxing(iptTgt)
      println("正在执行页面...",i,helper.stamp())
      //return
    }
    return
    // 添加字符集防止乱码
    val md = new model("com.mysql.jdbc.Driver","jdbc:mysql://localhost/zaueen?useUnicode=true&characterEncoding=UTF-8","root", "151001")
    //println(8888)
    //md.table("net_user")
    //val sqlInsert = "insert into `anjuke`(name,remark) values('数据新增测试','备注-Scala-插入数据')"
    //println("插入一条数据",md.update(sqlInsert))
    md.table("anjuke")
  }
  // 幽灵爬虫-
  def ghost(config:String,action:String):Unit = {
    val runTimeBegin = helper.detMsec()
    val br = "\r\n"
    val xconf = XML.loadFile(config)
    // 配置文件 -顶部元素
    val conf = xconf\"crawler"
    val top = conf\ action
    // 保存类型
    var saveType = (top\"@save").toString()
    if(saveType == ""){
      saveType = (conf\"@save").toString()
    }
    val ghostLog = (conf\"@log").toString()
    var timeOut = (top \ "@timeout").toString()
    if(timeOut != "false" && timeOut != "FALSE"){
      timeOut = (conf\"@timeout").toString()
    }else{
      timeOut = ""
    }
    var gtTimeOut = 0
    if(timeOut != ""){
      gtTimeOut = helper.str2int(timeOut)
    }

    var ghostLogTxt = br+br+"------->起始时间:"+helper.sysdate()
    var scvName = ""
    var saveText = ""
    if (saveType == "csv") {
      if ((top \ "@csvname").toString() != "") {
        scvName = (top \ "@csvname").toString()
      } else {
        scvName = action
      }
      scvName = "./runtime/" + scvName + "-" + helper.stamp() + ".csv"
    }
    // 仅仅一次
    var onlyOnce = 1
    var onlyOnceText = ""
    // 执行函数
    val actStart = (url: String) => {
      var doc = Jsoup.connect(url).get()
      if(gtTimeOut>0){
        doc = Jsoup.connect(url).timeout(gtTimeOut).get()
      }

      // 文件板保存写入
      //helper.put_content("./runtime/"+helper.stamp()+".html",doc.body().toString)
      val result = xconf \\ "result" // 普通数据解析
      val table = xconf\\"table"     // 表格解析引擎
      //var xmlData: Map[String, String] = Map("5555" -> "57shshsh")
      //val xmlData = new scala.collection.mutable.Map[String, String]
      // 变长数组
      val keys = new ArrayBuffer[String]()
      val desc = new ArrayBuffer[String]()
      val resultEngin = () => { // resultEngin 查询引擎
        var tmplen = 1
        for (rs <- result) {
          val scope = helper.xmlEscape(rs.attribute("scopen").last.toString)
          var selector = ""
          val pdivs = doc.select(scope)
          if (onlyOnce == 1) {
            onlyOnceText = br + "查询结构：" + scope + "("
          }
          for (pt <- pdivs) {
            var xmlLen = 0
            println("")
            for (nd <- rs \ "_") {
              if (nd.attribute("child") == "add") {
                var text = ""
                var ctt = ""
                for (add <- nd \ "_") {
                  try {
                    selector = helper.xmlEscape(add.attribute("text").last.toString)
                    if (onlyOnce == 1) {
                      onlyOnceText += selector + ","
                    }
                    text = pt.select(selector).text()
                    saveText += text + "-"
                    ctt += text
                    print(nd.label, ctt)
                  } catch {
                    case e: Exception => println(add.label + "无法从文件中解析，选择器如：" + add.attribute("text").last.toString)
                  }
                }
                if (tmplen == 1) {
                  keys += nd.label
                  desc += nd.text
                  //xmlData += (keys -> desc)
                }
                saveText += ","
              }
              else {
                if (tmplen == 1) {
                  keys += nd.label
                  desc += nd.text
                }
                var value = ""
                val parseStart = () => {
                  try {
                    selector = helper.xmlEscape(nd.attribute("text").last.toString)
                    if (onlyOnce == 1) {
                      onlyOnceText += selector + ","
                    }
                    value = pt.select(selector).text()
                    saveText += value
                    print(nd.label, value)
                  } catch {
                    case e: Exception => print(nd.label + "无法从文件中解析，选择器如：" + nd.attribute("text").last.toString + ">错误：" + e.getMessage)
                  }
                }
                // 依然无法避免 catch中断后面的程序
                parseStart()
                saveText += ","
              }
              xmlLen = xmlLen + 1
            }
            if (onlyOnce == 1) {
              onlyOnceText += ")" + br
            }
            println("")
            saveText += br
            tmplen = tmplen + 1
          }
        }
      }
      // table 数据查询引擎
      val tableEngin = () =>{
        var tmplen = 1
        val tableSel = (table\"@scopen").toString()
        var trs = doc.select("table tr")
        if(tableSel != "") trs = doc.select(helper.xmlEscape(tableSel))
        for(tr<- trs){
          for(k<- (table\"_")){
            val tb = tr.select(":nth-child("+k.attribute("td").last.toString()+")")
            if(tmplen == 1) {
              keys += k.label
              desc += k.text
            }
            saveText += tb.text()+","
          }
          println(saveText)
          saveText += br
          tmplen = tmplen + 1
        }
      }
      if(result.toString() != "") resultEngin()
      if(table.toString() != "") tableEngin()
      // 单页抓取处理结果
      println(keys)
      println(desc)
      if (saveType == "csv") {
        helper.file_append(scvName, saveText)
      }
      onlyOnce = onlyOnce +1
    }

    // 抓取执行控制台
    var url = (top \ "url").text
    // 起始页默认为1
    var pageBegin = (top \ "url" \ "@from").toString()
    if(pageBegin == "") pageBegin = "1"
    var pageEnd = (top \ "url" \ "@to").toString()
    var pageTpl = (top \ "url" \ "@tpl").toString()
    if(pageTpl == "") pageTpl = "__page__"
    // 自动生成最大的页码
    if(pageEnd == ""){
      ghostLogTxt += br+helper.systime()+">未发现最大页码，系统正在解析..."
      url = (top\"url"\"@url").toString()
      val toEl = top\"url"\"to"
      try{
        val attr = (toEl\"@attr").toString()
        val text = (toEl\"@text").toString()
        val selector = helper.xmlEscape((toEl\"@scopen").toString())
        val doc = Jsoup.connect(url.replaceAll(pageTpl,pageBegin)).get() // 从第一页中获取
        if(attr != "") pageEnd = doc.select(selector).attr(attr)
        if(text != "") pageEnd = doc.select(selector).text()
      }
      if(pageEnd == "") pageEnd = (toEl\"@refvalue").toString()
      if(!helper.is_numeric(pageEnd)){
        val arr = url.split(pageTpl)
        for(i<-arr){
          pageEnd = pageEnd.replaceAll(i,"")
        }
      }
      ghostLogTxt += br+helper.systime()+">通过解析获取到了最大页码:"+pageEnd
    }
    if (pageBegin != "" && pageEnd != "") {
      val min = helper.str2int(pageBegin)
      val max = helper.str2int(pageEnd)
      var newUrl = ""
      var runRpt = ""
      for (i <- min to max) {
        newUrl = url.replaceAll(pageTpl, i.toString)
        try {
          runRpt = helper.systime()+">"+newUrl + "正在爬取中..."
          println(runRpt)
          actStart(newUrl)
        } catch {
          case e: Exception => {
            runRpt = helper.systime()+">"+newUrl + "页面爬取失败？？ 错误提示："+e.getMessage
            println(runRpt)
          }
        }
        if(onlyOnce == 2) {
          ghostLogTxt += onlyOnceText
        }
        ghostLogTxt += br+runRpt+br+"本次所用时间："+helper.detMsec(runTimeBegin)+"(s)"
      }
      println("本次所用时间："+helper.detMsec(runTimeBegin)+"(s)")
      helper.file_append(ghostLog,ghostLogTxt)
    }
  }
}
